<!DOCTYPE html>





<html class="theme-next gemini" lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="generator" content="Hexo 3.9.0">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=7.3.0">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/CNY32x32.png?v=7.3.0">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/CNY16x16.png?v=7.3.0">
  <link rel="mask-icon" href="/images/logo.svg?v=7.3.0" color="#222">

<link rel="stylesheet" href="/css/main.css?v=7.3.0">


<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css?v=4.7.0">


<script id="hexo-configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Gemini',
    version: '7.3.0',
    exturl: false,
    sidebar: {"position":"left","display":"post","offset":12,"onmobile":false},
    back2top: {"enable":true,"sidebar":false,"scrollpercent":true},
    save_scroll: false,
    copycode: {"enable":true,"show_result":true,"style":null},
    fancybox: false,
    mediumzoom: false,
    lazyload: false,
    pangu: false,
    algolia: {
      appID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    },
    localsearch: {"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},
    path: 'search.xml',
    motion: {"enable":false,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    translation: {
      copy_button: '复制',
      copy_success: '复制成功',
      copy_failure: '复制失败'
    }
  };
</script>

  <meta name="description" content="Scrapy简介：Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架。 其可以应用在数据挖掘，信息处理或存储历史数据等一系列的程序中。其最初是为了页面抓取 (更确切来说, 网络抓取 )所设计的， 也可以应用在获取API所返回的数据(例如 Amazon Associates Web Services ) 或者通用的网络爬虫。Scrapy用途广泛，可以用于数据挖掘、监测和自动化测试。">
<meta name="keywords" content="爬虫">
<meta property="og:type" content="article">
<meta property="og:title" content="Python爬虫：基于Scrapy的爬取失踪人口数据小爬虫">
<meta property="og:url" content="localhost:4000/2019/06/01/Python爬虫：基于Scrapy的爬取失踪人口数据小爬虫/index.html">
<meta property="og:site_name" content="钟浩伟的个人博客">
<meta property="og:description" content="Scrapy简介：Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架。 其可以应用在数据挖掘，信息处理或存储历史数据等一系列的程序中。其最初是为了页面抓取 (更确切来说, 网络抓取 )所设计的， 也可以应用在获取API所返回的数据(例如 Amazon Associates Web Services ) 或者通用的网络爬虫。Scrapy用途广泛，可以用于数据挖掘、监测和自动化测试。">
<meta property="og:locale" content="zh-CN">
<meta property="og:image" content="https://zhong-blog.oss-cn-shenzhen.aliyuncs.com/blog/20190816235500.png!blog">
<meta property="og:image" content="">
<meta property="og:image" content="https://zhong-blog.oss-cn-shenzhen.aliyuncs.com/blog/20190910145638.png!blog">
<meta property="og:image" content="">
<meta property="og:image" content="https://zhong-blog.oss-cn-shenzhen.aliyuncs.com/blog/20190910145819.png!blog">
<meta property="og:image" content="">
<meta property="og:image" content="">
<meta property="og:image" content="https://zhong-blog.oss-cn-shenzhen.aliyuncs.com/blog/20190910145940.png!blog">
<meta property="og:updated_time" content="2019-09-10T07:00:35.649Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Python爬虫：基于Scrapy的爬取失踪人口数据小爬虫">
<meta name="twitter:description" content="Scrapy简介：Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架。 其可以应用在数据挖掘，信息处理或存储历史数据等一系列的程序中。其最初是为了页面抓取 (更确切来说, 网络抓取 )所设计的， 也可以应用在获取API所返回的数据(例如 Amazon Associates Web Services ) 或者通用的网络爬虫。Scrapy用途广泛，可以用于数据挖掘、监测和自动化测试。">
<meta name="twitter:image" content="https://zhong-blog.oss-cn-shenzhen.aliyuncs.com/blog/20190816235500.png!blog">
  <link rel="canonical" href="localhost:4000/2019/06/01/Python爬虫：基于Scrapy的爬取失踪人口数据小爬虫/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome: false,
    isPost: true,
    isPage: false,
    isArchive: false
  };
</script>

  <title>Python爬虫：基于Scrapy的爬取失踪人口数据小爬虫 | 钟浩伟的个人博客</title>
  








  <noscript>
  <style>
  .use-motion .motion-element,
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-title { opacity: initial; }

  .use-motion .logo,
  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-CN">

  <div class="container sidebar-position-left">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta">

    <div class="custom-logo-site-title">
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">钟浩伟的个人博客</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
        <p class="site-subtitle">欢迎光临</p>
      
  </div>

  <div class="site-nav-toggle">
    <button aria-label="切换导航栏">
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>


<nav class="site-nav">
  
  <ul id="menu" class="menu">
      
      
      
        
        <li class="menu-item menu-item-home">
      
    

    <a href="/" rel="section"><i class="menu-item-icon fa fa-fw fa-home"></i> <br>首页</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-tags">
      
    

    <a href="/tags/" rel="section"><i class="menu-item-icon fa fa-fw fa-tags"></i> <br>标签</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-categories">
      
    

    <a href="/categories/" rel="section"><i class="menu-item-icon fa fa-fw fa-th"></i> <br>分类</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-archives">
      
    

    <a href="/archives/" rel="section"><i class="menu-item-icon fa fa-fw fa-archive"></i> <br>归档</a>

  </li>
      <li class="menu-item menu-item-search">
        <a href="javascript:;" class="popup-trigger">
        
          <i class="menu-item-icon fa fa-search fa-fw"></i> <br>搜索</a>
      </li>
    
  </ul>

</nav>
  <div class="site-search">
    
  <div class="popup search-popup">
  <div class="search-header">
    <span class="search-icon">
      <i class="fa fa-search"></i>
    </span>
    <div class="search-input-wrapper">
      <input autocomplete="off" autocorrect="off" autocapitalize="none"
             placeholder="搜索..." spellcheck="false"
             type="text" id="search-input">
    </div>
    <span class="popup-btn-close">
      <i class="fa fa-times-circle"></i>
    </span>
  </div>
  <div id="search-result"></div>
</div>


  </div>
</div>
    </header>

    


    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
            

          <div id="content" class="content page-post-detail">
            

  <div id="posts" class="posts-expand">
    

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="localhost:4000/2019/06/01/Python爬虫：基于Scrapy的爬取失踪人口数据小爬虫/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="钟浩伟">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/profile.jpg">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="钟浩伟的个人博客">
    </span>
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">Python爬虫：基于Scrapy的爬取失踪人口数据小爬虫

            
          </h1>
        

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
                
              

              <time title="创建时间：2019-06-01 13:22:24" itemprop="dateCreated datePublished" datetime="2019-06-01T13:22:24+08:00">2019-06-01</time>
            </span>
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/categories/Python/" itemprop="url" rel="index"><span itemprop="name">Python</span></a></span>

                
                
              
            </span>
          

          

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <h1 id="Scrapy简介："><a href="#Scrapy简介：" class="headerlink" title="Scrapy简介："></a>Scrapy简介：</h1><p>Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架。 其可以应用在数据挖掘，信息处理或存储历史数据等一系列的程序中。其最初是为了页面抓取 (更确切来说, 网络抓取 )所设计的， 也可以应用在获取API所返回的数据(例如 Amazon Associates Web Services ) 或者通用的网络爬虫。Scrapy用途广泛，可以用于数据挖掘、监测和自动化测试。</p><a id="more"></a>
<p><strong>仓库地址：</strong></p>
<p><strong><a href="https://github.com/XiaoZhong233/missingPerson_spider" target="_blank" rel="noopener">https://github.com/XiaoZhong233/missingPerson_spider</a></strong></p>
<hr>
<h1 id="说明："><a href="#说明：" class="headerlink" title="说明："></a>说明：</h1><p>1、数据来源：宝贝回家网</p>
<p>2、数据后处理中加入了经纬度坐标，方便在空间上定位，原始数据是没有这个信息的</p>
<h1 id="技术点："><a href="#技术点：" class="headerlink" title="技术点："></a>技术点：</h1><ol>
<li>scrapy爬虫框架</li>
<li>mysql连接池技术</li>
<li>数据清洗（正则表达式）</li>
<li>百度API地理编码</li>
</ol>
<p><img src="https://zhong-blog.oss-cn-shenzhen.aliyuncs.com/blog/20190816235500.png!blog" alt="20190816235500"><img src="" alt="点击并拖拽以移动"></p>
<p>Spider的工作流程，就不具体说了，他是一个很强大的爬虫框架。</p>
<h1 id="运行结果："><a href="#运行结果：" class="headerlink" title="运行结果："></a><strong>运行结果：</strong></h1><p><img src="https://zhong-blog.oss-cn-shenzhen.aliyuncs.com/blog/20190910145638.png!blog" alt="20190910145638"><img src="" alt="点击并拖拽以移动"></p>
<p><img src="https://zhong-blog.oss-cn-shenzhen.aliyuncs.com/blog/20190910145819.png!blog" alt="20190910145819"><img src="" alt="点击并拖拽以移动"></p>
<h1 id="爬虫主要代码："><a href="#爬虫主要代码：" class="headerlink" title="爬虫主要代码："></a>爬虫主要代码：</h1><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br><span class="line">203</span><br><span class="line">204</span><br><span class="line">205</span><br><span class="line">206</span><br><span class="line">207</span><br><span class="line">208</span><br><span class="line">209</span><br><span class="line">210</span><br><span class="line">211</span><br><span class="line">212</span><br><span class="line">213</span><br><span class="line">214</span><br><span class="line">215</span><br><span class="line">216</span><br><span class="line">217</span><br><span class="line">218</span><br><span class="line">219</span><br><span class="line">220</span><br><span class="line">221</span><br><span class="line">222</span><br><span class="line">223</span><br><span class="line">224</span><br><span class="line">225</span><br><span class="line">226</span><br><span class="line">227</span><br><span class="line">228</span><br><span class="line">229</span><br><span class="line">230</span><br><span class="line">231</span><br><span class="line">232</span><br><span class="line">233</span><br><span class="line">234</span><br><span class="line">235</span><br><span class="line">236</span><br><span class="line">237</span><br><span class="line">238</span><br><span class="line">239</span><br><span class="line">240</span><br><span class="line">241</span><br><span class="line">242</span><br><span class="line">243</span><br><span class="line">244</span><br><span class="line">245</span><br><span class="line">246</span><br><span class="line">247</span><br><span class="line">248</span><br><span class="line">249</span><br><span class="line">250</span><br><span class="line">251</span><br><span class="line">252</span><br><span class="line">253</span><br><span class="line">254</span><br><span class="line">255</span><br><span class="line">256</span><br><span class="line">257</span><br><span class="line">258</span><br><span class="line">259</span><br><span class="line">260</span><br><span class="line">261</span><br><span class="line">262</span><br><span class="line">263</span><br><span class="line">264</span><br><span class="line">265</span><br><span class="line">266</span><br><span class="line">267</span><br><span class="line">268</span><br><span class="line">269</span><br><span class="line">270</span><br><span class="line">271</span><br><span class="line">272</span><br><span class="line">273</span><br><span class="line">274</span><br><span class="line">275</span><br><span class="line">276</span><br><span class="line">277</span><br><span class="line">278</span><br><span class="line">279</span><br><span class="line">280</span><br><span class="line">281</span><br><span class="line">282</span><br><span class="line">283</span><br><span class="line">284</span><br><span class="line">285</span><br><span class="line">286</span><br><span class="line">287</span><br><span class="line">288</span><br><span class="line">289</span><br><span class="line">290</span><br><span class="line">291</span><br><span class="line">292</span><br><span class="line">293</span><br><span class="line">294</span><br><span class="line">295</span><br><span class="line">296</span><br><span class="line">297</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> PersonItem <span class="keyword">import</span> PersonItem</span><br><span class="line"></span><br><span class="line">curPath = os.path.abspath(os.path.dirname(__file__))</span><br><span class="line">rootPath = os.path.split(curPath)[<span class="number">0</span>]</span><br><span class="line">sys.path.append(rootPath)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">from</span> urllib.request <span class="keyword">import</span> urlopen, quote</span><br><span class="line"><span class="keyword">import</span> datetime</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"></span><br><span class="line"><span class="comment">#E:\Desktop\python\zhiyuan_test\zhiyuan_test</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># PATH = os.environ</span></span><br><span class="line"><span class="comment"># for key in PATH:</span></span><br><span class="line"><span class="comment">#     print(key, PATH[key])</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#print(rootPath)</span></span><br><span class="line"><span class="comment">#百度坐标地理编码</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">getBlnglat</span><span class="params">(address)</span>:</span></span><br><span class="line">    url = <span class="string">'http://api.map.baidu.com/geocoder/v2/'</span></span><br><span class="line">    output = <span class="string">'json'</span></span><br><span class="line">    ak = <span class="string">'O8fpFPd6pw5VYK1uGsdMSAtL5MQRjAjs'</span></span><br><span class="line">    add = quote(address)  <span class="comment">#为防止乱码，先用quote进行编码</span></span><br><span class="line">    uri = url + <span class="string">'?'</span> + <span class="string">'address='</span> + add + <span class="string">'&amp;output='</span> + output + <span class="string">'&amp;ak='</span> + ak</span><br><span class="line">    req = urlopen(uri)</span><br><span class="line">    res = req.read().decode()</span><br><span class="line">    temp = json.loads(res)</span><br><span class="line">    <span class="comment"># print(temp.keys())</span></span><br><span class="line">    <span class="comment"># print(temp.values())</span></span><br><span class="line">    <span class="comment"># print(type(temp['result']))</span></span><br><span class="line">    <span class="keyword">if</span> temp[<span class="string">'status'</span>] == <span class="number">0</span>:</span><br><span class="line">        lng = temp[<span class="string">'result'</span>][<span class="string">'location'</span>][<span class="string">'lng'</span>]  <span class="comment"># 纬度</span></span><br><span class="line">        lat = (temp[<span class="string">'result'</span>][<span class="string">'location'</span>][<span class="string">'lat'</span>])  <span class="comment"># 经度</span></span><br><span class="line">        <span class="comment"># print(lng, lat)</span></span><br><span class="line">        <span class="keyword">return</span> str(lng) + <span class="string">","</span> + str(lat)</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        <span class="comment"># print(address+"没找到")</span></span><br><span class="line">        <span class="keyword">return</span> <span class="string">"NULL"</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">#爬取失踪人口信息</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MissingpersonSpider</span><span class="params">(scrapy.Spider)</span>:</span></span><br><span class="line">    name = <span class="string">'missingPerson'</span></span><br><span class="line">    allowed_domains = [<span class="string">'baobeihuijia.com'</span>]</span><br><span class="line">    <span class="comment">#https://bbs.baobeihuijia.com/thread-402350-1-1.html</span></span><br><span class="line">    start_urls = []</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">#读取本地数据</span></span><br><span class="line">    <span class="comment"># path = r".\missingUrl.json"</span></span><br><span class="line">    <span class="comment"># with open(path, 'rb') as f:</span></span><br><span class="line">    <span class="comment">#     data = json.load(f)</span></span><br><span class="line">    <span class="comment">#     #print(type(data))</span></span><br><span class="line">    <span class="comment"># len(data)</span></span><br><span class="line">    <span class="comment"># for index in range(6, 20):</span></span><br><span class="line">    <span class="comment">#     #str = re.match('.*', url)</span></span><br><span class="line">    <span class="comment">#     start_urls.append(data[index]["detailUrl"])</span></span><br><span class="line">    <span class="comment"># print(start_urls)</span></span><br><span class="line"></span><br><span class="line">    <span class="comment">#读取url数据库</span></span><br><span class="line">    <span class="keyword">import</span> MySQLdb</span><br><span class="line">    <span class="comment">#打开数据库连接</span></span><br><span class="line">    db = MySQLdb.connect(<span class="string">"localhost"</span>, <span class="string">"root"</span>, <span class="string">"root"</span>, <span class="string">"webgisdb"</span>, charset=<span class="string">'utf8'</span>)</span><br><span class="line">    cursor = db.cursor()</span><br><span class="line">    sql = <span class="string">"select * from url"</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        cursor.execute(sql)</span><br><span class="line">        results = cursor.fetchall()</span><br><span class="line">        <span class="keyword">for</span> row <span class="keyword">in</span> results:</span><br><span class="line">            start_urls.append(row[<span class="number">0</span>])</span><br><span class="line">            print(row[<span class="number">0</span>])</span><br><span class="line">    <span class="keyword">except</span>:</span><br><span class="line">        <span class="keyword">print</span></span><br><span class="line">        <span class="string">"Error: unable to fecth data"</span></span><br><span class="line">    db.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">string_toDatetime</span><span class="params">(st)</span>:</span></span><br><span class="line">        print(<span class="string">"2.把字符串转成datetime: "</span>, datetime.datetime.strptime(st, <span class="string">"%Y-%m-%d"</span>))</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span><span class="params">(self, response)</span>:</span></span><br><span class="line"></span><br><span class="line">        mycsv = [<span class="string">'NULL'</span>, <span class="string">'NULL'</span>, <span class="string">'NULL'</span>, <span class="string">'NULL'</span>, <span class="string">'NULL'</span>,<span class="string">"NULL"</span>, <span class="string">'NULL'</span>, <span class="string">'NULL'</span>,<span class="string">'NULL'</span>]  <span class="comment"># 初始化提取信息列表</span></span><br><span class="line">        <span class="comment"># 爬取图片url</span></span><br><span class="line">        imgurl=<span class="string">""</span></span><br><span class="line">        img = response.xpath(<span class="string">'//body//div[@class="t_fsz"]//ul/li//ignore_js_op/img/@file'</span>).extract();</span><br><span class="line">        <span class="keyword">if</span> len(img)&gt;<span class="number">0</span>:</span><br><span class="line">            <span class="comment">#print(img, "图片")</span></span><br><span class="line">            <span class="comment">#这里拉了多张图片</span></span><br><span class="line">            <span class="keyword">for</span> s <span class="keyword">in</span> img:</span><br><span class="line">                <span class="comment">#str = img[0]</span></span><br><span class="line">                rs=s.split(<span class="string">"forum"</span>)</span><br><span class="line">                tem = <span class="string">"https://youtu.baobeihuijia.com/forum"</span> + rs[<span class="number">1</span>]</span><br><span class="line">                <span class="keyword">if</span> imgurl==<span class="string">""</span>:</span><br><span class="line">                    imgurl=tem</span><br><span class="line">                <span class="keyword">else</span>:</span><br><span class="line">                    <span class="keyword">from</span> macpath <span class="keyword">import</span> join</span><br><span class="line">                    imgurl=join(<span class="string">","</span>.join((imgurl,tem)))</span><br><span class="line"></span><br><span class="line">            <span class="comment">#print(rs[1])</span></span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            imgurl=<span class="string">"http://www.svmuu.com/img/4.0/backstage/no-data.jpg"</span></span><br><span class="line">            <span class="keyword">pass</span></span><br><span class="line">        <span class="comment">#print(imgurl)</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">        <span class="comment">#爬取文字信息</span></span><br><span class="line">        <span class="keyword">for</span> li <span class="keyword">in</span> response.xpath(<span class="string">'//body//div[@class="t_fsz"]//ul/li'</span>):</span><br><span class="line"></span><br><span class="line">            <span class="comment"># 伪装</span></span><br><span class="line">            <span class="keyword">from</span> scrapy.conf <span class="keyword">import</span> settings</span><br><span class="line">            origin = <span class="string">"User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/"</span></span><br><span class="line">            ran = random.randint(<span class="number">10</span>, <span class="number">100000</span>)</span><br><span class="line">            origin += str(ran)</span><br><span class="line">            settings[<span class="string">'BOT_NAME'</span>] = origin</span><br><span class="line"></span><br><span class="line">            info = li.xpath(<span class="string">'font/text()'</span>).extract()</span><br><span class="line">            <span class="comment">#print(info)</span></span><br><span class="line"></span><br><span class="line">            <span class="keyword">if</span>(info!=<span class="literal">None</span>):</span><br><span class="line">                <span class="keyword">for</span> item <span class="keyword">in</span> info:</span><br><span class="line">                    <span class="keyword">if</span> <span class="string">"失踪日期"</span> <span class="keyword">in</span> item[<span class="number">0</span>:<span class="number">10</span>] <span class="keyword">or</span> <span class="string">"失踪日期"</span> <span class="keyword">in</span> item[<span class="number">0</span>:<span class="number">10</span>]:</span><br><span class="line">                        index = item.find(<span class="string">"："</span>)</span><br><span class="line">                        missingdate=item[index + <span class="number">1</span>:]</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">                        <span class="comment">#正则化匹配</span></span><br><span class="line">                        date_reg_exp = re.compile(<span class="string">'\d&#123;4&#125;[-/年]\d&#123;1,2&#125;[-/月]\d&#123;2&#125;\s*'</span>)</span><br><span class="line">                        matches_list = date_reg_exp.findall(missingdate)</span><br><span class="line">                        <span class="comment">#for match in matches_list:</span></span><br><span class="line">                        match = <span class="literal">None</span></span><br><span class="line">                        <span class="keyword">if</span> len(matches_list)==<span class="number">0</span>:</span><br><span class="line">                            <span class="comment">#print("正则表达式1无匹配"+missingdate+"，尝试2")</span></span><br><span class="line">                            date_reg_exp1= re.compile(<span class="string">'\d&#123;4&#125;[-/.,年]\d&#123;1,2&#125;[-/.,月]\s*'</span>)</span><br><span class="line">                            matches_list = date_reg_exp1.findall(missingdate)</span><br><span class="line">                            <span class="keyword">if</span> len(matches_list)==<span class="number">0</span>:</span><br><span class="line">                                <span class="comment">#print("正则表达式2无匹配"+missingdate+"，尝试3")</span></span><br><span class="line">                                date_reg_exp2 = re.compile(<span class="string">'\d&#123;4&#125;[年]\s*'</span>)</span><br><span class="line">                                matches_list = date_reg_exp2.findall(missingdate)</span><br><span class="line">                                <span class="keyword">if</span> len(matches_list)==<span class="number">0</span>:</span><br><span class="line">                                    <span class="comment">#print(missingdate+"无匹配")</span></span><br><span class="line">                                    match = <span class="literal">None</span></span><br><span class="line">                        <span class="keyword">else</span>:</span><br><span class="line">                            <span class="comment">#print(matches_list[0])</span></span><br><span class="line">                            match= matches_list[<span class="number">0</span>]</span><br><span class="line">                        times = <span class="literal">None</span></span><br><span class="line">                        <span class="comment">#解析日期</span></span><br><span class="line"></span><br><span class="line">                        <span class="keyword">if</span> match!=<span class="literal">None</span>:</span><br><span class="line">                            <span class="keyword">try</span>:</span><br><span class="line">                                times = time.strptime(match, <span class="string">"%Y年%m月%d"</span>)</span><br><span class="line">                            <span class="keyword">except</span> ValueError:</span><br><span class="line">                                <span class="keyword">try</span>:</span><br><span class="line">                                    times = time.strptime(match, <span class="string">"%Y-%m-%d"</span>)</span><br><span class="line">                                <span class="keyword">except</span> ValueError:</span><br><span class="line">                                    <span class="keyword">try</span>:</span><br><span class="line">                                        times = time.strptime(match, <span class="string">"%Y/%m/%d"</span>)</span><br><span class="line">                                    <span class="keyword">except</span> ValueError:</span><br><span class="line">                                        <span class="keyword">try</span>:</span><br><span class="line">                                            times = time.strptime(match, <span class="string">"%Y年%m月%d日 "</span>)</span><br><span class="line">                                        <span class="keyword">except</span> ValueError:</span><br><span class="line">                                            <span class="keyword">try</span>:</span><br><span class="line">                                                times = time.strptime(match, <span class="string">"%Y-%m-%d "</span>)</span><br><span class="line">                                            <span class="keyword">except</span> ValueError:</span><br><span class="line">                                                <span class="keyword">try</span>:</span><br><span class="line">                                                    times = time.strptime(match, <span class="string">"%Y/%m/%d "</span>)</span><br><span class="line">                                                <span class="keyword">except</span> ValueError:</span><br><span class="line">                                                    <span class="comment">#print("错误的日期格式或日期"+match)</span></span><br><span class="line">                                                    times=<span class="literal">None</span></span><br><span class="line"></span><br><span class="line">                        mycsv[<span class="number">8</span>] = times</span><br><span class="line">                        <span class="comment">#print(missingdate)</span></span><br><span class="line">                        <span class="keyword">pass</span></span><br><span class="line">                    <span class="keyword">if</span> <span class="string">"姓    名"</span> <span class="keyword">in</span> item[<span class="number">0</span>:<span class="number">6</span>]:</span><br><span class="line">                        index = item.find(<span class="string">"："</span>)</span><br><span class="line">                        missingName= item[index + <span class="number">1</span>:]</span><br><span class="line">                        mycsv[<span class="number">0</span>]= missingName</span><br><span class="line">                    <span class="keyword">if</span> <span class="string">"性    别"</span> <span class="keyword">in</span> item:</span><br><span class="line">                        index = item.find(<span class="string">"："</span>)</span><br><span class="line">                        sex=item[index + <span class="number">1</span>:]</span><br><span class="line">                        mycsv[<span class="number">1</span>] = sex</span><br><span class="line">                    <span class="keyword">if</span> <span class="string">"出生日期"</span> <span class="keyword">in</span> item:</span><br><span class="line">                        index = item.find(<span class="string">"："</span>)</span><br><span class="line">                        birth = item[index + <span class="number">1</span>:]</span><br><span class="line">                        <span class="comment">#正则化匹配</span></span><br><span class="line">                        date_reg_exp = re.compile(<span class="string">'\d&#123;4&#125;[-/年]\d&#123;1,2&#125;[-/月]\d&#123;2&#125;\s*'</span>)</span><br><span class="line">                        matches_list = date_reg_exp.findall(birth)</span><br><span class="line">                        <span class="comment">#for match in matches_list:</span></span><br><span class="line">                        match = <span class="literal">None</span></span><br><span class="line">                        <span class="keyword">if</span> len(matches_list) == <span class="number">0</span>:</span><br><span class="line">                            <span class="comment">#print("正则表达式1无匹配"+birth+"，尝试2")</span></span><br><span class="line">                            date_reg_exp1 = re.compile(<span class="string">'\d&#123;4&#125;[-/年]\d&#123;1,2&#125;[-/月]\s*'</span>)</span><br><span class="line">                            matches_list = date_reg_exp1.findall(birth)</span><br><span class="line">                            <span class="keyword">if</span> len(matches_list) == <span class="number">0</span>:</span><br><span class="line">                                <span class="comment">#print("正则表达式2无匹配"+birth+"，尝试3")</span></span><br><span class="line">                                date_reg_exp2 = re.compile(<span class="string">'\d&#123;4&#125;[年]\s*'</span>)</span><br><span class="line">                                matches_list = date_reg_exp2.findall(birth)</span><br><span class="line">                                <span class="keyword">if</span> len(matches_list) == <span class="number">0</span>:</span><br><span class="line">                                    <span class="comment">#print(birth+"无匹配")</span></span><br><span class="line">                                    match = <span class="literal">None</span></span><br><span class="line">                        <span class="keyword">else</span>:</span><br><span class="line">                            <span class="comment">#print(matches_list[0])</span></span><br><span class="line">                            match= matches_list[<span class="number">0</span>]</span><br><span class="line">                        times = <span class="literal">None</span></span><br><span class="line">                        <span class="comment">#解析日期</span></span><br><span class="line"></span><br><span class="line">                        <span class="keyword">if</span> match!=<span class="literal">None</span>:</span><br><span class="line">                            <span class="keyword">try</span>:</span><br><span class="line">                                times = time.strptime(match, <span class="string">"%Y年%m月%d"</span>)</span><br><span class="line">                            <span class="keyword">except</span> ValueError:</span><br><span class="line">                                <span class="keyword">try</span>:</span><br><span class="line">                                    times = time.strptime(match, <span class="string">"%Y-%m-%d"</span>)</span><br><span class="line">                                <span class="keyword">except</span> ValueError:</span><br><span class="line">                                    <span class="keyword">try</span>:</span><br><span class="line">                                        times = time.strptime(match, <span class="string">"%Y/%m/%d"</span>)</span><br><span class="line">                                    <span class="keyword">except</span> ValueError:</span><br><span class="line">                                        <span class="keyword">try</span>:</span><br><span class="line">                                            times = time.strptime(match, <span class="string">"%Y年%m月%d日 "</span>)</span><br><span class="line">                                        <span class="keyword">except</span> ValueError:</span><br><span class="line">                                            <span class="keyword">try</span>:</span><br><span class="line">                                                times = time.strptime(match, <span class="string">"%Y-%m-%d "</span>)</span><br><span class="line">                                            <span class="keyword">except</span> ValueError:</span><br><span class="line">                                                <span class="keyword">try</span>:</span><br><span class="line">                                                    times = time.strptime(match, <span class="string">"%Y/%m/%d"</span> )</span><br><span class="line">                                                <span class="keyword">except</span> ValueError:</span><br><span class="line">                                                    <span class="comment">#print("错误是日期格式或日期")</span></span><br><span class="line">                                                    times=<span class="literal">None</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">                        <span class="comment">#print(times)</span></span><br><span class="line"></span><br><span class="line">                        <span class="comment">#print(birth)</span></span><br><span class="line">                        mycsv[<span class="number">2</span>] = times</span><br><span class="line">                    <span class="keyword">if</span> <span class="string">"失踪时身高"</span> <span class="keyword">in</span> item:</span><br><span class="line">                        index = item.find(<span class="string">"："</span>)</span><br><span class="line">                        tall = item[index + <span class="number">1</span>:]</span><br><span class="line">                        print(<span class="string">"string:"</span> + tall)</span><br><span class="line">                        date_reg_exp = re.compile(<span class="string">'\d&#123;1,4&#125;'</span>)</span><br><span class="line">                        matches_list = date_reg_exp.findall(tall)</span><br><span class="line">                        <span class="comment">#print(matches_list)</span></span><br><span class="line">                        matchTall = <span class="literal">None</span></span><br><span class="line">                        <span class="keyword">if</span> len(matches_list) == <span class="number">0</span>:</span><br><span class="line">                            print(<span class="string">"身高正则匹配失败 string:"</span>+ tall)</span><br><span class="line">                        <span class="keyword">else</span>:</span><br><span class="line">                            matchTall=matches_list[<span class="number">0</span>]</span><br><span class="line">                            print(<span class="string">"正则化字符串： "</span>+matchTall)</span><br><span class="line">                        mycsv[<span class="number">3</span>] = matchTall</span><br><span class="line">                    <span class="keyword">if</span> <span class="string">"失踪地点"</span> <span class="keyword">in</span> item:</span><br><span class="line">                        index = item.find(<span class="string">"："</span>)</span><br><span class="line">                        adress = item[index + <span class="number">1</span>:]</span><br><span class="line">                        mycsv[<span class="number">4</span>] = adress</span><br><span class="line">                        mycsv[<span class="number">5</span>] = getBlnglat(adress)</span><br><span class="line">                    i<span class="string">f"失踪者特征描述"</span> <span class="keyword">in</span> item:</span><br><span class="line">                        index = item.find(<span class="string">"："</span>)</span><br><span class="line">                        describe = item[index + <span class="number">1</span>:]</span><br><span class="line">                        mycsv[<span class="number">6</span>] = describe</span><br><span class="line">                    pattern = re.compile(<span class="string">r'^\d+'</span>)</span><br><span class="line">                    detail = pattern.match(item)</span><br><span class="line">                    <span class="keyword">if</span> detail:</span><br><span class="line">                        tem = mycsv[<span class="number">7</span>]</span><br><span class="line">                        <span class="keyword">if</span> tem == <span class="string">"NULL"</span>:</span><br><span class="line">                            tem = <span class="string">""</span></span><br><span class="line">                        s = tem+item</span><br><span class="line">                        mycsv[<span class="number">7</span>] = s</span><br><span class="line"></span><br><span class="line">        <span class="comment">#and mycsv[1] != "NULL" and mycsv[2] != "NULL"</span></span><br><span class="line">        <span class="comment">#整合信息</span></span><br><span class="line">        <span class="keyword">if</span> mycsv[<span class="number">0</span>]!=<span class="string">"NULL"</span>  <span class="keyword">and</span> mycsv[<span class="number">4</span>]!=<span class="string">"NULL"</span> <span class="keyword">and</span> mycsv[<span class="number">5</span>]!=<span class="string">"NULL"</span> :</span><br><span class="line">            print(mycsv)</span><br><span class="line">            item = PersonItem()  <span class="comment"># 实例化item类</span></span><br><span class="line">            item[<span class="string">'name'</span>] = mycsv[<span class="number">0</span>]</span><br><span class="line">            item[<span class="string">'sex'</span>] = mycsv[<span class="number">1</span>]</span><br><span class="line">            item[<span class="string">'birth'</span>] = mycsv[<span class="number">2</span>]</span><br><span class="line">            item[<span class="string">'tall'</span>] = mycsv[<span class="number">3</span>]</span><br><span class="line">            item[<span class="string">'adress'</span>] = mycsv[<span class="number">4</span>]</span><br><span class="line">            item[<span class="string">'cor'</span>] = mycsv[<span class="number">5</span>]</span><br><span class="line">            item[<span class="string">'describe'</span>] = mycsv[<span class="number">6</span>]</span><br><span class="line">            item[<span class="string">'other'</span>] = mycsv[<span class="number">7</span>]</span><br><span class="line">            item[<span class="string">'imgsrc'</span>] = imgurl</span><br><span class="line">            item[<span class="string">'missingdate'</span>] = mycsv[<span class="number">8</span>]</span><br><span class="line">            <span class="keyword">yield</span> item</span><br></pre></td></tr></table></figure>

<p><img src="" alt="点击并拖拽以移动"></p>
<h1 id="最终结果："><a href="#最终结果：" class="headerlink" title="最终结果："></a>最终结果：</h1><p><img src="https://zhong-blog.oss-cn-shenzhen.aliyuncs.com/blog/20190910145940.png!blog" alt="20190910145940"></p>

    </div>

    
    
    
      
  <div class="popular-posts-header">相关文章推荐</div>
  <ul class="popular-posts">
    <li class="popular-posts-item">
      
      
      <div class="popular-posts-title"><a href="\2019\08\16\Scrapy爬取CSDN博客列表\" rel="bookmark">Scrapy爬取CSDN博客列表</a></div>
      
    </li>
  
  </ul>


    
    <div>
      
      <div>
    
        <div style="text-align:center;color: #ccc;font-size:14px;">-------------本文结束感谢您的阅读,如有疑问，请联系<a href="mailto:610596547@qq.com">610596547@qq.com</a>-------------</div>
    
</div>
      
    </div>
        
      

      <footer class="post-footer">
          
            
          
          <div class="post-tags">
            
              <a href="/tags/爬虫/" rel="tag"># 爬虫</a>
            
          </div>
        

        

          <div class="post-nav">
            <div class="post-nav-next post-nav-item">
              
                <a href="/2019/06/01/基于SSM的后台管理系统/" rel="next" title="基于SSM的后台管理系统">
                  <i class="fa fa-chevron-left"></i> 基于SSM的后台管理系统
                </a>
              
            </div>

            <span class="post-nav-divider"></span>

            <div class="post-nav-prev post-nav-item">
              
                <a href="/2019/06/01/GIS算法基础（九）矢量压缩算法-道格拉斯普克算法/" rel="prev" title="GIS算法基础（九）矢量压缩算法-道格拉斯普克算法">
                  GIS算法基础（九）矢量压缩算法-道格拉斯普克算法 <i class="fa fa-chevron-right"></i>
                </a>
              
            </div>
          </div>
        
      </footer>
    
  </div>
  
  
  
  </article>

  </div>


          </div>
          

        </div>
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-overview">

          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image"
      src="/images/profile.jpg"
      alt="钟浩伟">
  <p class="site-author-name" itemprop="name">钟浩伟</p>
  <div class="site-description motion-element" itemprop="description"></div>
</div>
  <nav class="site-state motion-element">
      <div class="site-state-item site-state-posts">
        
          <a href="/archives/">
        
          <span class="site-state-item-count">44</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
    
      
      
      <div class="site-state-item site-state-categories">
        
          
            <a href="/categories/">
          
        
        
        
          
        
          
        
          
        
          
        
          
        
          
        
        <span class="site-state-item-count">6</span>
        <span class="site-state-item-name">分类</span>
        </a>
      </div>
    
      
      
      <div class="site-state-item site-state-tags">
        
          
            <a href="/tags/">
          
        
        
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
        <span class="site-state-item-count">15</span>
        <span class="site-state-item-name">标签</span>
        </a>
      </div>
    
  </nav>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="https://github.com/XiaoZhong233" title="GitHub &rarr; https://github.com/XiaoZhong233" rel="noopener" target="_blank"><i class="fa fa-fw fa-github"></i>GitHub</a>
      </span>
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="mailto:601596547@qq.com" title="E-Mail &rarr; mailto:601596547@qq.com" rel="noopener" target="_blank"><i class="fa fa-fw fa-envelope"></i>E-Mail</a>
      </span>
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="https://blog.csdn.net/weixin_41154636" title="CSDN &rarr; https://blog.csdn.net/weixin_41154636" rel="noopener" target="_blank"><i class="fa fa-fw fa-book"></i>CSDN</a>
      </span>
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="http://profile.zhonghaowei.xyz/" title="Profile &rarr; http://profile.zhonghaowei.xyz/" rel="noopener" target="_blank"><i class="fa fa-fw fa-address-card-o"></i>Profile</a>
      </span>
    
  </div>



        </div>
      </div>
      <!--noindex-->
        <div class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
            
            
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#Scrapy简介："><span class="nav-number">1.</span> <span class="nav-text">Scrapy简介：</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#说明："><span class="nav-number">2.</span> <span class="nav-text">说明：</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#技术点："><span class="nav-number">3.</span> <span class="nav-text">技术点：</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#运行结果："><span class="nav-number">4.</span> <span class="nav-text">运行结果：</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#爬虫主要代码："><span class="nav-number">5.</span> <span class="nav-text">爬虫主要代码：</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#最终结果："><span class="nav-number">6.</span> <span class="nav-text">最终结果：</span></a></li></ol></div>
            

          </div>
        </div>
      <!--/noindex-->
      



    </div>
  </aside>
  <div id="sidebar-dimmer"></div>




      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; <span itemprop="copyrightYear">2020</span>
  <span class="with-love" id="animate">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">钟浩伟</span>
</div>


<div id="days"></div>
<div class="BbeiAn-info">
   	粤ICP备 -
    <a target="_blank" href="http://beian.miit.gov.cn" style="color:#f0d784"  rel="nofollow">粤ICP备19111001号-1</a> <!--a标签中增加nofollow属性，避免爬虫出站。-->| 
	<a target="_blank" href="http://www.beian.gov.cn/portal/registerSystemInfo?recordcode=" style="color:#f0d784;text-decoration:none;padding-left:30px;background:url(https://s1.ax1x.com/2018/09/29/ilmwIH.png) no-repeat left center" rel="nofollow">粤公网安备 粤ICP备19111001号-1</a>	  <!--这里将图标作为了背景，以使得能和后面的文字在同一行-->
</div>

<script>
function show_date_time(){
    window.setTimeout("show_date_time()", 1000);
    BirthDay=new Date("08/15/2019 22:40:14");
    today=new Date();
    timeold=(today.getTime()-BirthDay.getTime());
    sectimeold=timeold/1000
    secondsold=Math.floor(sectimeold);
    msPerDay=24*60*60*1000
    e_daysold=timeold/msPerDay
    daysold=Math.floor(e_daysold);
    e_hrsold=(e_daysold-daysold)*24;
    hrsold=setzero(Math.floor(e_hrsold));
    e_minsold=(e_hrsold-hrsold)*60;
    minsold=setzero(Math.floor((e_hrsold-hrsold)*60));
    seconds=setzero(Math.floor((e_minsold-minsold)*60));
    document.getElementById('days').innerHTML="<p>小破站已运行 "+daysold+" 天 "+hrsold+" 小时 "+minsold+" 分 "+seconds+" 秒</p>";
}
function setzero(i) {
    if (i<10) {
        i="0" + i
    };
    return i;
}
show_date_time();
</script>

        








        
      </div>
    </footer>
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
          <span id="scrollpercent"><span>0</span>%</span>
        
      </div>

    

  </div>

  
  <script src="/lib/jquery/index.js?v=3.4.1"></script>

<script src="/js/utils.js?v=7.3.0"></script>

<script src="/js/schemes/pisces.js?v=7.3.0"></script>



<script src="/js/next-boot.js?v=7.3.0"></script>




  















  <script src="/js/local-search.js?v=7.3.0"></script>










<script>
if (document.querySelectorAll('pre.mermaid').length) {
  NexT.utils.getScript('//cdn.jsdelivr.net/npm/mermaid@8/dist/mermaid.min.js', () => {
    mermaid.initialize({
      theme: 'default',
      logLevel: 3,
      flowchart: { curve: 'linear' },
      gantt: { axisFormat: '%m/%d/%Y' },
      sequence: { actorMargin: 50 }
    });
  }, window.mermaid);
}
</script>




  

  

  


  
  <script src="/js/scrollspy.js?v=7.3.0"></script>
<script src="/js/post-details.js?v=7.3.0"></script>



  
  <script type="text/javascript" src="//cdn.bootcss.com/canvas-nest.js/1.0.0/canvas-nest.min.js"></script>
  
<script src="/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"pluginRootPath":"live2dw/","pluginJsPath":"lib/","pluginModelPath":"assets/","tagMode":false,"log":false,"model":{"jsonPath":"/live2dw/assets/assets/hijiki.model.json"},"display":{"position":"right","width":150,"height":300},"mobile":{"show":false}});</script></body>
</html>
